package com.guiji.quartz.domain;

import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.wanghaomiao.xpath.model.JXDocument;
import com.guiji.quartz.config.ProxyConfig;
import com.guiji.quartz.task.DataSaveUtil;
import com.guiji.quartz.util.DelPicUtil;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @program: cms-vue-plus
 * @description: 不需要代理
 * @author: wangxw
 * @date 2021/11/29 10:21
 */
public class Bostonglobe {
	private static final Logger log = LoggerFactory.getLogger(Bostonglobe.class);

	public static void mainMethod() throws Exception {
		List<String> list = new ArrayList<>();
		list.add("https://www.bostonglobe.com/business/");
		list.add("https://www.bostonglobe.com/business/economy/?p1=BGMenu_Sections");
		list.forEach(url -> {
			HttpResponse response = null;
			try {
				response = HttpRequest.get(url)
					.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
					.header("Accept-Encoding", "gzip, deflate")
					.setConnectionTimeout(30000)
					.execute();

				String category = "Bostonglobe-EnglishBusiness";
				if (url.contains("economy")) {
					category = "bostonglobe-Economy";
				}
				if (null != response) {
					if (response.getStatus() == 200) {
						List<HttpCookie> cookies = response.getCookies();
						String resp = response.body();
						Set<String> resUrl = getUrl(resp);
						getContent(resUrl, cookies,category);
					} else {
						log.error("https://www.bostonglobe.com/business/" + String.valueOf(response.getStatus()));
					}
				} else {
					log.error("https://www.bostonglobe.com/business/" + String.valueOf(response.getStatus()));
				}

			} catch (Exception e) {
				log.error("访问 https://www.bostonglobe.com/business/ 失败" + e);
				return;
			}
		});




	}

	public static void setCookies(List<HttpCookie> cookies, String host) throws MalformedURLException, URISyntaxException {
		if (null != cookies && cookies.size() > 0) {
			CookieManager ccm = new CookieManager();
			CookieStore CookieStore = ccm.getCookieStore();
			for (HttpCookie cookie : cookies) {
				URL domain = new URL(host);
				CookieStore.add(domain.toURI(), cookie);
			}
			HttpRequest.setCookieManager(ccm);
		}
	}

	private static void getContent(Set<String> listContent, List<HttpCookie> cookies,String category) throws MalformedURLException, URISyntaxException, InterruptedException {
		for (String url : listContent) {
			Thread.sleep(1000);

			if (!url.startsWith("http")) {
				url = new StringBuffer("https://www.bostonglobe.com").append(url).toString();
			}

			setCookies(cookies, "https://www.bostonglobe.com");

			log.info("getContent执行到-{},listContent长度-{}", url, listContent.size());
			String doc;
			JXDocument jxDocument;
			try {
				HttpResponse response = HttpRequest.get(url).setConnectionTimeout(15000).execute();
				doc = response.body();
				jxDocument = new JXDocument(doc);
			} catch (Exception e) {
				log.info("代理失败2" + e);
				continue;
			}
			String title = null;
			try {
				String patternTitle = "\"articleHeadline\":\"(.*?)\"";
				Pattern pTitle = Pattern.compile(patternTitle);
				Matcher mTitle = pTitle.matcher(doc);
				if (mTitle.find()) {
					title = mTitle.group(1);
				}
			} catch (Exception e) {
				log.error("可能是付费内容");
				continue;
			}
			StringBuffer content = new StringBuffer();
			String contentStr = null;
			JXDocument jxDocumentP;
			Document document = null;
			try {
				int start = doc.indexOf("<main");
				int end = doc.indexOf("</main>");

				contentStr = doc.substring(start, end + 7);
				document = Jsoup.parse(contentStr);
				Element mainArea = document.getElementById("article-body");
				//mainArea.select("div[class='newsletter-shortcode']").remove();
				Elements pElements = mainArea.select("P");
				if (null != pElements && pElements.size() > 0) {
					for (Element pdom : pElements) {
						jxDocumentP = new JXDocument(pdom.toString());
						String temp = jxDocumentP.sel("//text()").get(0).toString();
						if (null != temp && !temp.contains("ALSO READ|") && !temp.contains("READ MORE|")) {
							content.append(temp + "$$$");
						}
					}
				}

				contentStr = content.toString();
			} catch (Exception e) {
				continue;
			}

			List<Object> srcImgs = new ArrayList<>();
			List<String> newImgs = new ArrayList<>();
			String img = null;
			try {

				//"originalUrl":"
				String pattern = "\"originalUrl\":\"(.*?)\"";
				Pattern p = Pattern.compile(pattern);
				Matcher m = p.matcher(doc);
				if (m.find()) {
					img = m.group(1);
					srcImgs.add(img);

				}

			} catch (Exception e) {
				log.error("可能没有图片");
				img = "";
			} finally {
				if (srcImgs.size() > 0) {
					newImgs = DelPicUtil.delPic(srcImgs);
				}
			}

			Date date = new Date();
			if (contentStr.length() > 50000 || contentStr.replaceAll("\\$|[\\s\\p{Zs}]", "").length() < 20 || newImgs.size() > 20) {
				continue;
			}
			try {
				System.out.println("=======================================================");
				System.out.println(url);
				System.out.println(title);

				for (Object imgitem : srcImgs) {
					System.out.println(" " + imgitem);
				}
				System.out.println(contentStr);
				System.out.println("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++");
				DataSaveUtil.saveData(title, contentStr, newImgs, url, date, "https://www.bostonglobe.com/business/", category);
				log.info(Thread.currentThread().getName() + "-----------------{}----------------",category);
			} catch (Exception e) {
				continue;
			}

		}
	}

	private static Set<String> getUrl(String content) {
		Set<String> res = new HashSet<>();
		String pattern = "_url\":\"/[0-9]{4}/[0-9]{2}/[0-9]{2}/business/.*?/\"";
		Pattern p = Pattern.compile(pattern);
		Matcher m = p.matcher(content);
		while (m.find()) {
			String temp = m.group();
			temp = temp.replaceAll("_url\":\"", "");
			temp = temp.replaceAll("\"", "");
			res.add(temp);
		}

		return res;
	}


//	public static void main(String[] args) throws Exception {
//		mainMethod();
//	}

}


